package com.chance.cc.crawler.development.scripts.pcauto.article;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.tags.CategoryTag;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * @author lt
 * @version 1.0
 * @date 2021-01-11 16:55:31
 * @email okprog@sina.com
 */
public class PcAutoArticleKeyWordsSyncScript extends CrawlerCommonScript {
    private Logger logger = LoggerFactory.getLogger(PcAutoArticleKeyWordsSyncScript.class);

    public static final String indexRegex = "https?://www\\.pcauto\\.com\\.cn/sitemap/sitemap\\.html";

    private static final String scriptSite = "article_keywords";

    @Override
    public String domain() {
        return "pcauto";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(indexRegex);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();
        String crawlerSite = categoryTag.getKVTagStrVal("site");
        return crawlerSite.equalsIgnoreCase(scriptSite);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        return null;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CategoryTag categoryTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag();
        String domain = categoryTag.getKVTagStrVal("domain");
        String site = categoryTag.getKVTagStrVal("site");
        try {
            List<String> kwLines = IOUtils.readLines(new FileInputStream("/data/chance_crawler_test/data/pcauto/pcAutoArticleKeys.txt"));
            int count = 0;
            for (String kwLine : kwLines) {
                count++;
                JSONObject kwObj = JSONObject.parseObject(kwLine);
                String pid = kwObj.getString("pid");
                String mid = kwObj.getString("mid");
                String type1st = kwObj.getString("pingdao");
                String type2nd = kwObj.getString("mokuai");
                String url = kwObj.getString("mokuai_url");

                Map<String,String> contentMap = new HashMap<>();
                contentMap.put("type1st",type1st);
                contentMap.put("type2nd",type2nd);
                contentMap.put("url",url);
                contentMap.put("dataId", StringUtils.joinWith("-",domain,site,pid,mid));

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(StringUtils.joinWith("-",domain,site,"keyword",pid,mid))
                        .releaseTime(System.currentTimeMillis())
                        .build();

                crawlerData.setCrawlerContent(JSON.toJSONString(contentMap));
                crawlerDataList.add(crawlerData);
            }
            logger.info("init " + domain + " " + site + " total: " + count);
        }catch (Exception e){
            logger.error(e.getMessage(),e);
        }
        return crawlerDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }
}
